#!/usr/bin/env python3
# -*- coding: utf-8 -*-


__author__ = 'Minho Lee'

import asyncio,logging,re,sys
from bs4 import BeautifulSoup
import requests


def log(sql, args=()):
    logging.info('SQL: %s' % sql)


def request_catalog(uri):

    headers ={
        'User-Agent' : r'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
        'Referer': r'http://www.lagou.com/zhaopin/Python/?labelWords=label',
        'Connection': 'keep-alive'
    }
    # req = request.Request(uri,headers=headers)
    #
    # response = request.urlopen(req).read()
    #
    # #content = str(response,'utf-8')
    # typeEncode = sys.getfilesystemencoding()
    #
    # content = response.decode("gbk",errors='ignore').encode('utf-8').decode('utf-8')

    # with open('./a.txt', 'w', encoding='utf-8') as f:
    #     f.write(content)

    req = requests.get(uri,headers = headers)

    res = req.text.encode(req.encoding)

    soup = BeautifulSoup(res, 'html.parser')

    soup_list = soup.select('.abook_contents_list')

    content = []
    for x in soup_list[0].find_all('li') :
        a_el = x.find('a')
        if a_el is not None:
            content.append(a_el.string)

    return content

